# -*- coding:utf-8 -*-
# --author： jingfeng 
# time: 2019/1/23



import requests
from fake_useragent import UserAgent
from lxml import etree
import json

ua = UserAgent()
headers = {'User-Agent': ua.random}
import os


def get_teacher_info():
    for page in range(1, 56):
        url = 'https://edu.csdn.net/lecturer?page=%s' % (str(page))

        # 得到 姓名 课程数   学生数   介绍地址
        response = requests.get(url, headers=headers)
        response.encoding = response.apparent_encoding
        html = response.text
        selector = etree.HTML(html)

        name = selector.xpath('//dl[@class="lector_list"]/dd/ul/li[1]/a/text()')
        course_num = selector.xpath('//dl[@class="lector_list"]/dd/ul/li['
                                    '2]/span/text()')
        student_num = selector.xpath('//dl[@class="lector_list"]/dd/ul/li['
                                     '3]/span/text()')
        info_url = selector.xpath('//dl[@class="lector_list"]/dd/ul/li['
                                  '1]/a/@href')

        infolist = []

        for url in info_url:
            print(url)
            response = requests.get(url, headers=headers)
            response.encoding = response.apparent_encoding
            html = response.text
            selector = etree.HTML(html)
            # 擅长领域
            field = selector.xpath('//div[@class="lector1_intro"]/p[1]/text('
                                   ')')[0]
            company = selector.xpath('//div[@class="lector1_intro"]/p[2]'
                                     '/a/text()')
            if company:
                info = selector.xpath('//div[@class="lector1_intro"]/p['
                                      '3]/text()')
                if info:
                    info = info[0]
                else:
                    info = ''
            else:
                company = ''
                info = selector.xpath('//div[@class="lector1_intro"]/p['
                                      '2]/text()')[0]
            infolist.append({'field': field, 'company': company, 'info': info})

        for i in range(len(name)):
            list = [name[i], course_num[i], student_num[i], infolist[i].get(
                'field'), infolist[i].get('company'), infolist[i].get('info')]

            save_info(list)
        print('%s页' % str(page))


import csv

file = open('teacher.csv', 'wt', encoding='utf-8', newline='')
writer = csv.writer(file)

writer.writerow(['name', 'course_num', 'student_num', 'field', 'company',
                 'info'])


def save_info(info):
    writer.writerow(info)


get_teacher_info()

file.close()
